#!/usr/bin/env python
#-*- coding: utf-8 -*-
# -------------------------------------------
# Filename:    get_proxy_url.py
# Revision:    1.0
# Date:        2012-4-8
# Author:      simon-zzm
# WEB:         www.simonzhang.net
# Email:       simon-zzm@163.com
# -------------------------------------------
import os
import urllib,urllib2
import re
import socket

#### base  set
socket.setdefaulttimeout(30)
proxy_list = []

def get_proxy_ip(url):
    try:
        user_agent = "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"
        values = {'name' : 'Michael Foord', 'location' : 'Northampton', 'language' : 'Python' }
        headers = {'User-Agent' : user_agent }
        data = urllib.urlencode(values)
        req = urllib2.Request(url, data, headers)
        html_file=urllib2.urlopen(req).read()
    except:
        return
    find_proxy_ip_z=re.compile(r'''\d{2,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}</.*?>\d{2,4}|\d{2,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,4}''',re.DOTALL)
    list_proxy_ip=find_proxy_ip_z.findall(html_file)
    filter_ip(list_proxy_ip)

def filter_ip(list_proxy_ip):
        for pro_add in list_proxy_ip:
                count_l=re.search('[</=-a-zA-Z]{2,4}',pro_add)
                if count_l is not None:
                        big_proxy_ip=re.match('(\d{2,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})<.*?>(\d{2,4})',pro_add,re.DOTALL)
                        big_proxy_ip.group()
                        proxy_list.append(('%s:%s')%(big_proxy_ip.group(1),big_proxy_ip.group(2)))
                else:
                        proxy_list.append(pro_add)

def main():
    try:
        os.remove('proxy_ip.txt')
    except:
        print 'Not find file'
    #delete proxy_ip.txt
    get_address=open('address.txt','r')
    get_address_list=get_address.readlines()
    #read get proxy url
    for gal in get_address_list:
        if gal[-1] == '\n':
            gal = gal[0:-1]
            print gal
            get_proxy_ip(gal)
        else:
            print gal
            get_proxy_ip(gal)
    get_address.close()
  
    #filter double proxy Ip
    filter_pro_ip=set(proxy_list)
    f_pro_ip=open('proxy_ip.txt','w')
    for f_i in filter_pro_ip:
        if f_i[-1]=='\n':
            tpm_f_i=f_i[0:-1]
            print >> f_pro_ip,tpm_f_i
        else:
            print >> f_pro_ip,f_i
    f_pro_ip.close()
    print 'Game over!!!'

if __name__ == '__main__':  
    main()
